{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "60c44bf2-e234-4567-9cc3-c79a88994162",
   "metadata": {},
   "source": [
    "<strong>获取epchan网站的m代码</strong>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4e3caed6-e99f-4a38-961f-f77b7d9ea3fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1d95403b-64ac-461b-b0ea-c9972ca51aa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [401]>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.epchan.com/book2/index.php?dir=Utilities'\n",
    "requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "449d69bd-11c3-4594-8f36-e7d588a03607",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用webdriver\n",
    "from selenium import webdriver  # 导入 Selenium 库，用于模拟浏览器操作\n",
    "import requests  # 导入 requests 库，用于发送 HTTP 请求\n",
    "import re\n",
    "from bs4 import BeautifulSoup  # 导入 BeautifulSoup 库，用于解析 HTML\n",
    "import html.parser  # 导入 html.parser 模块\n",
    "import time  # 导入 time 模块，用于暂停执行\n",
    "import pandas as pd  # 导入 pandas 库，用于数据处理和分析\n",
    "import undetected_chromedriver as uc\n",
    "import random\n",
    "# 创建 ChromeOptions 对象并设置浏览器选项\n",
    "options = uc.ChromeOptions()\n",
    "options.add_argument(\"--no-sandbox\")\n",
    "options.add_argument(\"--disable-dev-shm-usage\")\n",
    "options.add_argument(\"--window-size=444x133\")\n",
    "options.add_argument(\"--disable-gpu\")\n",
    "options.add_argument(\"--hide-scrollbars\")\n",
    "# options.add_argument(\"--blink-settings=imagesEnabled=false\")  # 不加载图片\n",
    "# options.add_argument(\"--headless\")  # 无界面模式\n",
    "driver = uc.Chrome(options)\n",
    "url =  'https://www.epchan.com/book2/index.php?dir=Utilities' # 目标网页的 URL\n",
    "driver.get(url)  # 打开目标网页，使用浏览器访问"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "ad54963f-458a-412b-bacb-8257277af75c",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_file_names = []\n",
    "page_content = driver.page_source\n",
    "soup = BeautifulSoup(page_content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "7b2fa477-7a0c-4de0-b883-c6ee51049065",
   "metadata": {},
   "outputs": [],
   "source": [
    "#_f('parseBlbDivsSplitsCalendar.m',939,'Jun-19-22','Utilities/parseBlbDivsSplitsCalendar.m',1655661833);\n",
    "list_file_names = re.findall(\"_f\\\\('(.*?.m)'\",str(soup))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "320e52ac-629c-4b2a-b7ff-236b5e0f38f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "for file_name in list_file_names:\n",
    "    download_url = 'https://www.epchan.com/book2/Utilities/'+file_name\n",
    "    driver.get(download_url)\n",
    "    download_page_content = driver.page_source\n",
    "    fun_text = re.findall(r\"<pre[^>]*>(.*?)</pre>\", str(download_page_content), re.DOTALL)\n",
    "    fun_text = fun_text[0]\n",
    "    with open('algotrading/Utilities/'+file_name, \"w\") as f:\n",
    "        f.write(fun_text)\n",
    "    time.sleep(1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
